services:
  metastore:
    image: postgres:11
    hostname: metastore
    environment:
      POSTGRES_PASSWORD: jupyter
    ports:
      - "5432:5432"
    volumes:
      - metastore:/var/lib/postgresql/data
      - ./ddl/init.sql:/docker-entrypoint-initdb.d/init.sql
    restart: always
    networks:
      sparknet:
        ipv4_address: 172.28.1.1
    extra_hosts:
      - "master:172.28.1.2"
      - "worker1:172.28.1.3"
      - "worker2:172.28.1.4"
      - "history:172.28.1.5"

  master:
    image: hadoop-hive-spark-master
    hostname: master
    depends_on:
      - metastore
    environment:
      SPARK_MASTER_HOST: 172.28.1.2
      SPARK_LOCAL_IP: 172.28.1.2
      SPARK_LOCAL_HOSTNAME: master
    ports:
      - "4040:4040"
      - "8020:8020"
      - "8080:8080"
      - "8088:8088"
      - "9870:9870"
      - "10000:10000"
    volumes:
      - namenode:/opt/hadoop/dfs/name
    restart: always
    networks:
      sparknet:
        ipv4_address: 172.28.1.2
    extra_hosts:
      - "metastore:172.28.1.1"
      - "worker1:172.28.1.3"
      - "worker2:172.28.1.4"
      - "history:172.28.1.5"

  worker1:
    image: hadoop-hive-spark-worker
    hostname: worker1
    depends_on:
      - master
    environment:
      SPARK_MASTER_HOST: 172.28.1.2
      SPARK_LOCAL_IP: 172.28.1.3
      SPARK_LOCAL_HOSTNAME: worker1
    ports:
      - "8042:8042"
      - "8081:8081"
      - "9864:9864"
    volumes:
      - datanode1:/opt/hadoop/dfs/data
    restart: always
    networks:
      sparknet:
        ipv4_address: 172.28.1.3
    extra_hosts:
      - "metastore:172.28.1.1"
      - "master:172.28.1.2"
      - "worker2:172.28.1.4"
      - "history:172.28.1.5"

  worker2:
    image: hadoop-hive-spark-worker
    hostname: worker2
    depends_on:
      - master
    environment:
      SPARK_MASTER_HOST: 172.28.1.2
      SPARK_LOCAL_IP: 172.28.1.4
      SPARK_LOCAL_HOSTNAME: worker2
    ports:
      - "8043:8042"
      - "8082:8081"
      - "9865:9864"
    volumes:
      - datanode2:/opt/hadoop/dfs/data
    restart: always
    networks:
      sparknet:
        ipv4_address: 172.28.1.4
    extra_hosts:
      - "metastore:172.28.1.1"
      - "master:172.28.1.2"
      - "worker1:172.28.1.3"
      - "history:172.28.1.5"

  history:
    image: hadoop-hive-spark-history
    hostname: history
    depends_on:
      - master
      - worker1
      - worker2
    environment:
      SPARK_MASTER_HOST: 172.28.1.2
      SPARK_LOCAL_IP: 172.28.1.5
      SPARK_LOCAL_HOSTNAME: history
    ports:
      - "18080:18080"
      - "19888:19888"
    restart: always
    networks:
      sparknet:
        ipv4_address: 172.28.1.5
    extra_hosts:
      - "metastore:172.28.1.1"
      - "master:172.28.1.2"
      - "worker1:172.28.1.3"
      - "worker2:172.28.1.4"

  jupyter:
    image: hadoop-hive-spark-jupyter
    hostname: jupyter
    environment:
      SPARK_MASTER_HOST: 172.28.1.2
      SPARK_LOCAL_IP: 172.28.1.6
      SPARK_LOCAL_HOSTNAME: jupyter
    depends_on:
      - master
      - worker1
      - worker2
    ports:
      - "8888:8888"
    volumes:
      - ./jupyter/notebook:/home/jupyter
    restart: always
    networks:
      sparknet:
        ipv4_address: 172.28.1.6
    extra_hosts:
      - "metastore:172.28.1.1"
      - "master:172.28.1.2"
      - "worker1:172.28.1.3"
      - "worker2:172.28.1.4"
      - "history:172.28.1.5"
    # deploy:
    #   resources:
    #     reservations:
    #       devices:
    #         - capabilities: [gpu]

  dev:
    image: hadoop-hive-spark-dev
    hostname: dev
    environment:
      SPARK_MASTER_HOST: 172.28.1.2
      SPARK_LOCAL_IP: 172.28.1.7
      SPARK_LOCAL_HOSTNAME: dev
    volumes:
      - ./dev/home:/home/jupyter
    networks:
      sparknet:
        ipv4_address: 172.28.1.7
    # deploy:
      # resources:
      #   reservations:
      #     devices:
      #       - capabilities: [gpu]

volumes:
  namenode:
  namesecondary:
  datanode1:
  datanode2:
  metastore:

networks:
  sparknet:
    ipam:
      driver: default
      config:
        - subnet: 172.28.0.0/16